{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0867d158",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/lib/python3/dist-packages/scipy/__init__.py:146: UserWarning: A NumPy version >=1.17.3 and <1.25.0 is required for this version of SciPy (detected version 1.26.4\n",
      "  warnings.warn(f\"A NumPy version >={np_minversion} and <{np_maxversion}\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2025-06-14 20:33:43,509] [INFO] [real_accelerator.py:239:get_accelerator] Setting ds_accelerator to cuda (auto detect)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/bin/ld: cannot find -laio: No such file or directory\n",
      "collect2: error: ld returned 1 exit status\n",
      "/usr/bin/ld: cannot find -laio: No such file or directory\n",
      "collect2: error: ld returned 1 exit status\n"
     ]
    }
   ],
   "source": [
    "from qwen_model import Model, WhisperEncoder\n",
    "from safetensors import safe_open\n",
    "from transformers import AutoConfig, AutoProcessor, AutoTokenizer, AddedToken, AutoFeatureExtractor\n",
    "from datasets import Audio\n",
    "from transformers import TextStreamer\n",
    "from qwen_model import Model, WhisperEncoder\n",
    "import transformers\n",
    "import torch\n",
    "import math"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "8ee980e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "WhisperEncoder.register_for_auto_class(\"AutoModel\")\n",
    "Model.register_for_auto_class(\"AutoModelForCausalLM\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3431a7e7",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "checkpoint-100\tcheckpoint-200\tcheckpoint-300\truns\r\n"
     ]
    }
   ],
   "source": [
    "!ls lora-embedding-64-audio-qwen2.5-7b-malaysian-10k-stage2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "78023c7c",
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_extractor = AutoFeatureExtractor.from_pretrained('openai/whisper-large-v3')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ef4db5b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained('Qwen/Qwen2.5-7B-Instruct')\n",
    "chat_template = \"{% set audio_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\\nYou are a helpful assistant.<|im_end|>\\n{% endif %}<|im_start|>{{ message['role'] }}\\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\\n{% else %}{% for content in message['content'] %}{% if 'audio' in content or 'audio_url' in content or message['type'] == 'audio' %}{% set audio_count.value = audio_count.value + 1 %}Audio {{ audio_count.value }}: <|audio_bos|><|AUDIO|><|audio_eos|>\\n{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\\n{% endif %}\"\n",
    "tokenizer.chat_template = chat_template\n",
    "audio_token = \"<|AUDIO|>\"\n",
    "audio_bos_token = \"<|audio_bos|>\"\n",
    "audio_eos_token = \"<|audio_eos|>\"\n",
    "new_tokens = [AddedToken(audio_token), AddedToken(audio_bos_token), AddedToken(audio_eos_token)]\n",
    "tokenizer.add_tokens(new_tokens)\n",
    "audio_token_id = tokenizer.vocab[audio_token]\n",
    "pad_token_id = tokenizer.pad_token_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "3d30ffeb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "151665"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "audio_token_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cef566d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "streamer = TextStreamer(tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "dace4620",
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoint = 'lora-embedding-64-audio-qwen2.5-7b-malaysian-10k-stage2/checkpoint-300'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8a7005c0",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Sliding Window Attention is enabled but not implemented for `sdpa`; unexpected results may be encountered.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aa0736c522a546b783e91bf9cff7dab2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/4 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at lora-embedding-64-audio-qwen2.5-7b-malaysian-10k-stage2/checkpoint-300 were not used when initializing Model: ['model.layers.0.mlp.down_proj.linear.weight', 'model.layers.0.mlp.down_proj.lora_A.weight', 'model.layers.0.mlp.down_proj.lora_B.weight', 'model.layers.0.mlp.gate_proj.linear.weight', 'model.layers.0.mlp.gate_proj.lora_A.weight', 'model.layers.0.mlp.gate_proj.lora_B.weight', 'model.layers.0.mlp.up_proj.linear.weight', 'model.layers.0.mlp.up_proj.lora_A.weight', 'model.layers.0.mlp.up_proj.lora_B.weight', 'model.layers.0.self_attn.k_proj.linear.bias', 'model.layers.0.self_attn.k_proj.linear.weight', 'model.layers.0.self_attn.k_proj.lora_A.weight', 'model.layers.0.self_attn.k_proj.lora_B.weight', 'model.layers.0.self_attn.o_proj.linear.weight', 'model.layers.0.self_attn.o_proj.lora_A.weight', 'model.layers.0.self_attn.o_proj.lora_B.weight', 'model.layers.0.self_attn.q_proj.linear.bias', 'model.layers.0.self_attn.q_proj.linear.weight', 'model.layers.0.self_attn.q_proj.lora_A.weight', 'model.layers.0.self_attn.q_proj.lora_B.weight', 'model.layers.0.self_attn.v_proj.linear.bias', 'model.layers.0.self_attn.v_proj.linear.weight', 'model.layers.0.self_attn.v_proj.lora_A.weight', 'model.layers.0.self_attn.v_proj.lora_B.weight', 'model.layers.1.mlp.down_proj.linear.weight', 'model.layers.1.mlp.down_proj.lora_A.weight', 'model.layers.1.mlp.down_proj.lora_B.weight', 'model.layers.1.mlp.gate_proj.linear.weight', 'model.layers.1.mlp.gate_proj.lora_A.weight', 'model.layers.1.mlp.gate_proj.lora_B.weight', 'model.layers.1.mlp.up_proj.linear.weight', 'model.layers.1.mlp.up_proj.lora_A.weight', 'model.layers.1.mlp.up_proj.lora_B.weight', 'model.layers.1.self_attn.k_proj.linear.bias', 'model.layers.1.self_attn.k_proj.linear.weight', 'model.layers.1.self_attn.k_proj.lora_A.weight', 'model.layers.1.self_attn.k_proj.lora_B.weight', 'model.layers.1.self_attn.o_proj.linear.weight', 'model.layers.1.self_attn.o_proj.lora_A.weight', 'model.layers.1.self_attn.o_proj.lora_B.weight', 'model.layers.1.self_attn.q_proj.linear.bias', 'model.layers.1.self_attn.q_proj.linear.weight', 'model.layers.1.self_attn.q_proj.lora_A.weight', 'model.layers.1.self_attn.q_proj.lora_B.weight', 'model.layers.1.self_attn.v_proj.linear.bias', 'model.layers.1.self_attn.v_proj.linear.weight', 'model.layers.1.self_attn.v_proj.lora_A.weight', 'model.layers.1.self_attn.v_proj.lora_B.weight', 'model.layers.10.mlp.down_proj.linear.weight', 'model.layers.10.mlp.down_proj.lora_A.weight', 'model.layers.10.mlp.down_proj.lora_B.weight', 'model.layers.10.mlp.gate_proj.linear.weight', 'model.layers.10.mlp.gate_proj.lora_A.weight', 'model.layers.10.mlp.gate_proj.lora_B.weight', 'model.layers.10.mlp.up_proj.linear.weight', 'model.layers.10.mlp.up_proj.lora_A.weight', 'model.layers.10.mlp.up_proj.lora_B.weight', 'model.layers.10.self_attn.k_proj.linear.bias', 'model.layers.10.self_attn.k_proj.linear.weight', 'model.layers.10.self_attn.k_proj.lora_A.weight', 'model.layers.10.self_attn.k_proj.lora_B.weight', 'model.layers.10.self_attn.o_proj.linear.weight', 'model.layers.10.self_attn.o_proj.lora_A.weight', 'model.layers.10.self_attn.o_proj.lora_B.weight', 'model.layers.10.self_attn.q_proj.linear.bias', 'model.layers.10.self_attn.q_proj.linear.weight', 'model.layers.10.self_attn.q_proj.lora_A.weight', 'model.layers.10.self_attn.q_proj.lora_B.weight', 'model.layers.10.self_attn.v_proj.linear.bias', 'model.layers.10.self_attn.v_proj.linear.weight', 'model.layers.10.self_attn.v_proj.lora_A.weight', 'model.layers.10.self_attn.v_proj.lora_B.weight', 'model.layers.11.mlp.down_proj.linear.weight', 'model.layers.11.mlp.down_proj.lora_A.weight', 'model.layers.11.mlp.down_proj.lora_B.weight', 'model.layers.11.mlp.gate_proj.linear.weight', 'model.layers.11.mlp.gate_proj.lora_A.weight', 'model.layers.11.mlp.gate_proj.lora_B.weight', 'model.layers.11.mlp.up_proj.linear.weight', 'model.layers.11.mlp.up_proj.lora_A.weight', 'model.layers.11.mlp.up_proj.lora_B.weight', 'model.layers.11.self_attn.k_proj.linear.bias', 'model.layers.11.self_attn.k_proj.linear.weight', 'model.layers.11.self_attn.k_proj.lora_A.weight', 'model.layers.11.self_attn.k_proj.lora_B.weight', 'model.layers.11.self_attn.o_proj.linear.weight', 'model.layers.11.self_attn.o_proj.lora_A.weight', 'model.layers.11.self_attn.o_proj.lora_B.weight', 'model.layers.11.self_attn.q_proj.linear.bias', 'model.layers.11.self_attn.q_proj.linear.weight', 'model.layers.11.self_attn.q_proj.lora_A.weight', 'model.layers.11.self_attn.q_proj.lora_B.weight', 'model.layers.11.self_attn.v_proj.linear.bias', 'model.layers.11.self_attn.v_proj.linear.weight', 'model.layers.11.self_attn.v_proj.lora_A.weight', 'model.layers.11.self_attn.v_proj.lora_B.weight', 'model.layers.12.mlp.down_proj.linear.weight', 'model.layers.12.mlp.down_proj.lora_A.weight', 'model.layers.12.mlp.down_proj.lora_B.weight', 'model.layers.12.mlp.gate_proj.linear.weight', 'model.layers.12.mlp.gate_proj.lora_A.weight', 'model.layers.12.mlp.gate_proj.lora_B.weight', 'model.layers.12.mlp.up_proj.linear.weight', 'model.layers.12.mlp.up_proj.lora_A.weight', 'model.layers.12.mlp.up_proj.lora_B.weight', 'model.layers.12.self_attn.k_proj.linear.bias', 'model.layers.12.self_attn.k_proj.linear.weight', 'model.layers.12.self_attn.k_proj.lora_A.weight', 'model.layers.12.self_attn.k_proj.lora_B.weight', 'model.layers.12.self_attn.o_proj.linear.weight', 'model.layers.12.self_attn.o_proj.lora_A.weight', 'model.layers.12.self_attn.o_proj.lora_B.weight', 'model.layers.12.self_attn.q_proj.linear.bias', 'model.layers.12.self_attn.q_proj.linear.weight', 'model.layers.12.self_attn.q_proj.lora_A.weight', 'model.layers.12.self_attn.q_proj.lora_B.weight', 'model.layers.12.self_attn.v_proj.linear.bias', 'model.layers.12.self_attn.v_proj.linear.weight', 'model.layers.12.self_attn.v_proj.lora_A.weight', 'model.layers.12.self_attn.v_proj.lora_B.weight', 'model.layers.13.mlp.down_proj.linear.weight', 'model.layers.13.mlp.down_proj.lora_A.weight', 'model.layers.13.mlp.down_proj.lora_B.weight', 'model.layers.13.mlp.gate_proj.linear.weight', 'model.layers.13.mlp.gate_proj.lora_A.weight', 'model.layers.13.mlp.gate_proj.lora_B.weight', 'model.layers.13.mlp.up_proj.linear.weight', 'model.layers.13.mlp.up_proj.lora_A.weight', 'model.layers.13.mlp.up_proj.lora_B.weight', 'model.layers.13.self_attn.k_proj.linear.bias', 'model.layers.13.self_attn.k_proj.linear.weight', 'model.layers.13.self_attn.k_proj.lora_A.weight', 'model.layers.13.self_attn.k_proj.lora_B.weight', 'model.layers.13.self_attn.o_proj.linear.weight', 'model.layers.13.self_attn.o_proj.lora_A.weight', 'model.layers.13.self_attn.o_proj.lora_B.weight', 'model.layers.13.self_attn.q_proj.linear.bias', 'model.layers.13.self_attn.q_proj.linear.weight', 'model.layers.13.self_attn.q_proj.lora_A.weight', 'model.layers.13.self_attn.q_proj.lora_B.weight', 'model.layers.13.self_attn.v_proj.linear.bias', 'model.layers.13.self_attn.v_proj.linear.weight', 'model.layers.13.self_attn.v_proj.lora_A.weight', 'model.layers.13.self_attn.v_proj.lora_B.weight', 'model.layers.14.mlp.down_proj.linear.weight', 'model.layers.14.mlp.down_proj.lora_A.weight', 'model.layers.14.mlp.down_proj.lora_B.weight', 'model.layers.14.mlp.gate_proj.linear.weight', 'model.layers.14.mlp.gate_proj.lora_A.weight', 'model.layers.14.mlp.gate_proj.lora_B.weight', 'model.layers.14.mlp.up_proj.linear.weight', 'model.layers.14.mlp.up_proj.lora_A.weight', 'model.layers.14.mlp.up_proj.lora_B.weight', 'model.layers.14.self_attn.k_proj.linear.bias', 'model.layers.14.self_attn.k_proj.linear.weight', 'model.layers.14.self_attn.k_proj.lora_A.weight', 'model.layers.14.self_attn.k_proj.lora_B.weight', 'model.layers.14.self_attn.o_proj.linear.weight', 'model.layers.14.self_attn.o_proj.lora_A.weight', 'model.layers.14.self_attn.o_proj.lora_B.weight', 'model.layers.14.self_attn.q_proj.linear.bias', 'model.layers.14.self_attn.q_proj.linear.weight', 'model.layers.14.self_attn.q_proj.lora_A.weight', 'model.layers.14.self_attn.q_proj.lora_B.weight', 'model.layers.14.self_attn.v_proj.linear.bias', 'model.layers.14.self_attn.v_proj.linear.weight', 'model.layers.14.self_attn.v_proj.lora_A.weight', 'model.layers.14.self_attn.v_proj.lora_B.weight', 'model.layers.15.mlp.down_proj.linear.weight', 'model.layers.15.mlp.down_proj.lora_A.weight', 'model.layers.15.mlp.down_proj.lora_B.weight', 'model.layers.15.mlp.gate_proj.linear.weight', 'model.layers.15.mlp.gate_proj.lora_A.weight', 'model.layers.15.mlp.gate_proj.lora_B.weight', 'model.layers.15.mlp.up_proj.linear.weight', 'model.layers.15.mlp.up_proj.lora_A.weight', 'model.layers.15.mlp.up_proj.lora_B.weight', 'model.layers.15.self_attn.k_proj.linear.bias', 'model.layers.15.self_attn.k_proj.linear.weight', 'model.layers.15.self_attn.k_proj.lora_A.weight', 'model.layers.15.self_attn.k_proj.lora_B.weight', 'model.layers.15.self_attn.o_proj.linear.weight', 'model.layers.15.self_attn.o_proj.lora_A.weight', 'model.layers.15.self_attn.o_proj.lora_B.weight', 'model.layers.15.self_attn.q_proj.linear.bias', 'model.layers.15.self_attn.q_proj.linear.weight', 'model.layers.15.self_attn.q_proj.lora_A.weight', 'model.layers.15.self_attn.q_proj.lora_B.weight', 'model.layers.15.self_attn.v_proj.linear.bias', 'model.layers.15.self_attn.v_proj.linear.weight', 'model.layers.15.self_attn.v_proj.lora_A.weight', 'model.layers.15.self_attn.v_proj.lora_B.weight', 'model.layers.16.mlp.down_proj.linear.weight', 'model.layers.16.mlp.down_proj.lora_A.weight', 'model.layers.16.mlp.down_proj.lora_B.weight', 'model.layers.16.mlp.gate_proj.linear.weight', 'model.layers.16.mlp.gate_proj.lora_A.weight', 'model.layers.16.mlp.gate_proj.lora_B.weight', 'model.layers.16.mlp.up_proj.linear.weight', 'model.layers.16.mlp.up_proj.lora_A.weight', 'model.layers.16.mlp.up_proj.lora_B.weight', 'model.layers.16.self_attn.k_proj.linear.bias', 'model.layers.16.self_attn.k_proj.linear.weight', 'model.layers.16.self_attn.k_proj.lora_A.weight', 'model.layers.16.self_attn.k_proj.lora_B.weight', 'model.layers.16.self_attn.o_proj.linear.weight', 'model.layers.16.self_attn.o_proj.lora_A.weight', 'model.layers.16.self_attn.o_proj.lora_B.weight', 'model.layers.16.self_attn.q_proj.linear.bias', 'model.layers.16.self_attn.q_proj.linear.weight', 'model.layers.16.self_attn.q_proj.lora_A.weight', 'model.layers.16.self_attn.q_proj.lora_B.weight', 'model.layers.16.self_attn.v_proj.linear.bias', 'model.layers.16.self_attn.v_proj.linear.weight', 'model.layers.16.self_attn.v_proj.lora_A.weight', 'model.layers.16.self_attn.v_proj.lora_B.weight', 'model.layers.17.mlp.down_proj.linear.weight', 'model.layers.17.mlp.down_proj.lora_A.weight', 'model.layers.17.mlp.down_proj.lora_B.weight', 'model.layers.17.mlp.gate_proj.linear.weight', 'model.layers.17.mlp.gate_proj.lora_A.weight', 'model.layers.17.mlp.gate_proj.lora_B.weight', 'model.layers.17.mlp.up_proj.linear.weight', 'model.layers.17.mlp.up_proj.lora_A.weight', 'model.layers.17.mlp.up_proj.lora_B.weight', 'model.layers.17.self_attn.k_proj.linear.bias', 'model.layers.17.self_attn.k_proj.linear.weight', 'model.layers.17.self_attn.k_proj.lora_A.weight', 'model.layers.17.self_attn.k_proj.lora_B.weight', 'model.layers.17.self_attn.o_proj.linear.weight', 'model.layers.17.self_attn.o_proj.lora_A.weight', 'model.layers.17.self_attn.o_proj.lora_B.weight', 'model.layers.17.self_attn.q_proj.linear.bias', 'model.layers.17.self_attn.q_proj.linear.weight', 'model.layers.17.self_attn.q_proj.lora_A.weight', 'model.layers.17.self_attn.q_proj.lora_B.weight', 'model.layers.17.self_attn.v_proj.linear.bias', 'model.layers.17.self_attn.v_proj.linear.weight', 'model.layers.17.self_attn.v_proj.lora_A.weight', 'model.layers.17.self_attn.v_proj.lora_B.weight', 'model.layers.18.mlp.down_proj.linear.weight', 'model.layers.18.mlp.down_proj.lora_A.weight', 'model.layers.18.mlp.down_proj.lora_B.weight', 'model.layers.18.mlp.gate_proj.linear.weight', 'model.layers.18.mlp.gate_proj.lora_A.weight', 'model.layers.18.mlp.gate_proj.lora_B.weight', 'model.layers.18.mlp.up_proj.linear.weight', 'model.layers.18.mlp.up_proj.lora_A.weight', 'model.layers.18.mlp.up_proj.lora_B.weight', 'model.layers.18.self_attn.k_proj.linear.bias', 'model.layers.18.self_attn.k_proj.linear.weight', 'model.layers.18.self_attn.k_proj.lora_A.weight', 'model.layers.18.self_attn.k_proj.lora_B.weight', 'model.layers.18.self_attn.o_proj.linear.weight', 'model.layers.18.self_attn.o_proj.lora_A.weight', 'model.layers.18.self_attn.o_proj.lora_B.weight', 'model.layers.18.self_attn.q_proj.linear.bias', 'model.layers.18.self_attn.q_proj.linear.weight', 'model.layers.18.self_attn.q_proj.lora_A.weight', 'model.layers.18.self_attn.q_proj.lora_B.weight', 'model.layers.18.self_attn.v_proj.linear.bias', 'model.layers.18.self_attn.v_proj.linear.weight', 'model.layers.18.self_attn.v_proj.lora_A.weight', 'model.layers.18.self_attn.v_proj.lora_B.weight', 'model.layers.19.mlp.down_proj.linear.weight', 'model.layers.19.mlp.down_proj.lora_A.weight', 'model.layers.19.mlp.down_proj.lora_B.weight', 'model.layers.19.mlp.gate_proj.linear.weight', 'model.layers.19.mlp.gate_proj.lora_A.weight', 'model.layers.19.mlp.gate_proj.lora_B.weight', 'model.layers.19.mlp.up_proj.linear.weight', 'model.layers.19.mlp.up_proj.lora_A.weight', 'model.layers.19.mlp.up_proj.lora_B.weight', 'model.layers.19.self_attn.k_proj.linear.bias', 'model.layers.19.self_attn.k_proj.linear.weight', 'model.layers.19.self_attn.k_proj.lora_A.weight', 'model.layers.19.self_attn.k_proj.lora_B.weight', 'model.layers.19.self_attn.o_proj.linear.weight', 'model.layers.19.self_attn.o_proj.lora_A.weight', 'model.layers.19.self_attn.o_proj.lora_B.weight', 'model.layers.19.self_attn.q_proj.linear.bias', 'model.layers.19.self_attn.q_proj.linear.weight', 'model.layers.19.self_attn.q_proj.lora_A.weight', 'model.layers.19.self_attn.q_proj.lora_B.weight', 'model.layers.19.self_attn.v_proj.linear.bias', 'model.layers.19.self_attn.v_proj.linear.weight', 'model.layers.19.self_attn.v_proj.lora_A.weight', 'model.layers.19.self_attn.v_proj.lora_B.weight', 'model.layers.2.mlp.down_proj.linear.weight', 'model.layers.2.mlp.down_proj.lora_A.weight', 'model.layers.2.mlp.down_proj.lora_B.weight', 'model.layers.2.mlp.gate_proj.linear.weight', 'model.layers.2.mlp.gate_proj.lora_A.weight', 'model.layers.2.mlp.gate_proj.lora_B.weight', 'model.layers.2.mlp.up_proj.linear.weight', 'model.layers.2.mlp.up_proj.lora_A.weight', 'model.layers.2.mlp.up_proj.lora_B.weight', 'model.layers.2.self_attn.k_proj.linear.bias', 'model.layers.2.self_attn.k_proj.linear.weight', 'model.layers.2.self_attn.k_proj.lora_A.weight', 'model.layers.2.self_attn.k_proj.lora_B.weight', 'model.layers.2.self_attn.o_proj.linear.weight', 'model.layers.2.self_attn.o_proj.lora_A.weight', 'model.layers.2.self_attn.o_proj.lora_B.weight', 'model.layers.2.self_attn.q_proj.linear.bias', 'model.layers.2.self_attn.q_proj.linear.weight', 'model.layers.2.self_attn.q_proj.lora_A.weight', 'model.layers.2.self_attn.q_proj.lora_B.weight', 'model.layers.2.self_attn.v_proj.linear.bias', 'model.layers.2.self_attn.v_proj.linear.weight', 'model.layers.2.self_attn.v_proj.lora_A.weight', 'model.layers.2.self_attn.v_proj.lora_B.weight', 'model.layers.20.mlp.down_proj.linear.weight', 'model.layers.20.mlp.down_proj.lora_A.weight', 'model.layers.20.mlp.down_proj.lora_B.weight', 'model.layers.20.mlp.gate_proj.linear.weight', 'model.layers.20.mlp.gate_proj.lora_A.weight', 'model.layers.20.mlp.gate_proj.lora_B.weight', 'model.layers.20.mlp.up_proj.linear.weight', 'model.layers.20.mlp.up_proj.lora_A.weight', 'model.layers.20.mlp.up_proj.lora_B.weight', 'model.layers.20.self_attn.k_proj.linear.bias', 'model.layers.20.self_attn.k_proj.linear.weight', 'model.layers.20.self_attn.k_proj.lora_A.weight', 'model.layers.20.self_attn.k_proj.lora_B.weight', 'model.layers.20.self_attn.o_proj.linear.weight', 'model.layers.20.self_attn.o_proj.lora_A.weight', 'model.layers.20.self_attn.o_proj.lora_B.weight', 'model.layers.20.self_attn.q_proj.linear.bias', 'model.layers.20.self_attn.q_proj.linear.weight', 'model.layers.20.self_attn.q_proj.lora_A.weight', 'model.layers.20.self_attn.q_proj.lora_B.weight', 'model.layers.20.self_attn.v_proj.linear.bias', 'model.layers.20.self_attn.v_proj.linear.weight', 'model.layers.20.self_attn.v_proj.lora_A.weight', 'model.layers.20.self_attn.v_proj.lora_B.weight', 'model.layers.21.mlp.down_proj.linear.weight', 'model.layers.21.mlp.down_proj.lora_A.weight', 'model.layers.21.mlp.down_proj.lora_B.weight', 'model.layers.21.mlp.gate_proj.linear.weight', 'model.layers.21.mlp.gate_proj.lora_A.weight', 'model.layers.21.mlp.gate_proj.lora_B.weight', 'model.layers.21.mlp.up_proj.linear.weight', 'model.layers.21.mlp.up_proj.lora_A.weight', 'model.layers.21.mlp.up_proj.lora_B.weight', 'model.layers.21.self_attn.k_proj.linear.bias', 'model.layers.21.self_attn.k_proj.linear.weight', 'model.layers.21.self_attn.k_proj.lora_A.weight', 'model.layers.21.self_attn.k_proj.lora_B.weight', 'model.layers.21.self_attn.o_proj.linear.weight', 'model.layers.21.self_attn.o_proj.lora_A.weight', 'model.layers.21.self_attn.o_proj.lora_B.weight', 'model.layers.21.self_attn.q_proj.linear.bias', 'model.layers.21.self_attn.q_proj.linear.weight', 'model.layers.21.self_attn.q_proj.lora_A.weight', 'model.layers.21.self_attn.q_proj.lora_B.weight', 'model.layers.21.self_attn.v_proj.linear.bias', 'model.layers.21.self_attn.v_proj.linear.weight', 'model.layers.21.self_attn.v_proj.lora_A.weight', 'model.layers.21.self_attn.v_proj.lora_B.weight', 'model.layers.22.mlp.down_proj.linear.weight', 'model.layers.22.mlp.down_proj.lora_A.weight', 'model.layers.22.mlp.down_proj.lora_B.weight', 'model.layers.22.mlp.gate_proj.linear.weight', 'model.layers.22.mlp.gate_proj.lora_A.weight', 'model.layers.22.mlp.gate_proj.lora_B.weight', 'model.layers.22.mlp.up_proj.linear.weight', 'model.layers.22.mlp.up_proj.lora_A.weight', 'model.layers.22.mlp.up_proj.lora_B.weight', 'model.layers.22.self_attn.k_proj.linear.bias', 'model.layers.22.self_attn.k_proj.linear.weight', 'model.layers.22.self_attn.k_proj.lora_A.weight', 'model.layers.22.self_attn.k_proj.lora_B.weight', 'model.layers.22.self_attn.o_proj.linear.weight', 'model.layers.22.self_attn.o_proj.lora_A.weight', 'model.layers.22.self_attn.o_proj.lora_B.weight', 'model.layers.22.self_attn.q_proj.linear.bias', 'model.layers.22.self_attn.q_proj.linear.weight', 'model.layers.22.self_attn.q_proj.lora_A.weight', 'model.layers.22.self_attn.q_proj.lora_B.weight', 'model.layers.22.self_attn.v_proj.linear.bias', 'model.layers.22.self_attn.v_proj.linear.weight', 'model.layers.22.self_attn.v_proj.lora_A.weight', 'model.layers.22.self_attn.v_proj.lora_B.weight', 'model.layers.23.mlp.down_proj.linear.weight', 'model.layers.23.mlp.down_proj.lora_A.weight', 'model.layers.23.mlp.down_proj.lora_B.weight', 'model.layers.23.mlp.gate_proj.linear.weight', 'model.layers.23.mlp.gate_proj.lora_A.weight', 'model.layers.23.mlp.gate_proj.lora_B.weight', 'model.layers.23.mlp.up_proj.linear.weight', 'model.layers.23.mlp.up_proj.lora_A.weight', 'model.layers.23.mlp.up_proj.lora_B.weight', 'model.layers.23.self_attn.k_proj.linear.bias', 'model.layers.23.self_attn.k_proj.linear.weight', 'model.layers.23.self_attn.k_proj.lora_A.weight', 'model.layers.23.self_attn.k_proj.lora_B.weight', 'model.layers.23.self_attn.o_proj.linear.weight', 'model.layers.23.self_attn.o_proj.lora_A.weight', 'model.layers.23.self_attn.o_proj.lora_B.weight', 'model.layers.23.self_attn.q_proj.linear.bias', 'model.layers.23.self_attn.q_proj.linear.weight', 'model.layers.23.self_attn.q_proj.lora_A.weight', 'model.layers.23.self_attn.q_proj.lora_B.weight', 'model.layers.23.self_attn.v_proj.linear.bias', 'model.layers.23.self_attn.v_proj.linear.weight', 'model.layers.23.self_attn.v_proj.lora_A.weight', 'model.layers.23.self_attn.v_proj.lora_B.weight', 'model.layers.24.mlp.down_proj.linear.weight', 'model.layers.24.mlp.down_proj.lora_A.weight', 'model.layers.24.mlp.down_proj.lora_B.weight', 'model.layers.24.mlp.gate_proj.linear.weight', 'model.layers.24.mlp.gate_proj.lora_A.weight', 'model.layers.24.mlp.gate_proj.lora_B.weight', 'model.layers.24.mlp.up_proj.linear.weight', 'model.layers.24.mlp.up_proj.lora_A.weight', 'model.layers.24.mlp.up_proj.lora_B.weight', 'model.layers.24.self_attn.k_proj.linear.bias', 'model.layers.24.self_attn.k_proj.linear.weight', 'model.layers.24.self_attn.k_proj.lora_A.weight', 'model.layers.24.self_attn.k_proj.lora_B.weight', 'model.layers.24.self_attn.o_proj.linear.weight', 'model.layers.24.self_attn.o_proj.lora_A.weight', 'model.layers.24.self_attn.o_proj.lora_B.weight', 'model.layers.24.self_attn.q_proj.linear.bias', 'model.layers.24.self_attn.q_proj.linear.weight', 'model.layers.24.self_attn.q_proj.lora_A.weight', 'model.layers.24.self_attn.q_proj.lora_B.weight', 'model.layers.24.self_attn.v_proj.linear.bias', 'model.layers.24.self_attn.v_proj.linear.weight', 'model.layers.24.self_attn.v_proj.lora_A.weight', 'model.layers.24.self_attn.v_proj.lora_B.weight', 'model.layers.25.mlp.down_proj.linear.weight', 'model.layers.25.mlp.down_proj.lora_A.weight', 'model.layers.25.mlp.down_proj.lora_B.weight', 'model.layers.25.mlp.gate_proj.linear.weight', 'model.layers.25.mlp.gate_proj.lora_A.weight', 'model.layers.25.mlp.gate_proj.lora_B.weight', 'model.layers.25.mlp.up_proj.linear.weight', 'model.layers.25.mlp.up_proj.lora_A.weight', 'model.layers.25.mlp.up_proj.lora_B.weight', 'model.layers.25.self_attn.k_proj.linear.bias', 'model.layers.25.self_attn.k_proj.linear.weight', 'model.layers.25.self_attn.k_proj.lora_A.weight', 'model.layers.25.self_attn.k_proj.lora_B.weight', 'model.layers.25.self_attn.o_proj.linear.weight', 'model.layers.25.self_attn.o_proj.lora_A.weight', 'model.layers.25.self_attn.o_proj.lora_B.weight', 'model.layers.25.self_attn.q_proj.linear.bias', 'model.layers.25.self_attn.q_proj.linear.weight', 'model.layers.25.self_attn.q_proj.lora_A.weight', 'model.layers.25.self_attn.q_proj.lora_B.weight', 'model.layers.25.self_attn.v_proj.linear.bias', 'model.layers.25.self_attn.v_proj.linear.weight', 'model.layers.25.self_attn.v_proj.lora_A.weight', 'model.layers.25.self_attn.v_proj.lora_B.weight', 'model.layers.26.mlp.down_proj.linear.weight', 'model.layers.26.mlp.down_proj.lora_A.weight', 'model.layers.26.mlp.down_proj.lora_B.weight', 'model.layers.26.mlp.gate_proj.linear.weight', 'model.layers.26.mlp.gate_proj.lora_A.weight', 'model.layers.26.mlp.gate_proj.lora_B.weight', 'model.layers.26.mlp.up_proj.linear.weight', 'model.layers.26.mlp.up_proj.lora_A.weight', 'model.layers.26.mlp.up_proj.lora_B.weight', 'model.layers.26.self_attn.k_proj.linear.bias', 'model.layers.26.self_attn.k_proj.linear.weight', 'model.layers.26.self_attn.k_proj.lora_A.weight', 'model.layers.26.self_attn.k_proj.lora_B.weight', 'model.layers.26.self_attn.o_proj.linear.weight', 'model.layers.26.self_attn.o_proj.lora_A.weight', 'model.layers.26.self_attn.o_proj.lora_B.weight', 'model.layers.26.self_attn.q_proj.linear.bias', 'model.layers.26.self_attn.q_proj.linear.weight', 'model.layers.26.self_attn.q_proj.lora_A.weight', 'model.layers.26.self_attn.q_proj.lora_B.weight', 'model.layers.26.self_attn.v_proj.linear.bias', 'model.layers.26.self_attn.v_proj.linear.weight', 'model.layers.26.self_attn.v_proj.lora_A.weight', 'model.layers.26.self_attn.v_proj.lora_B.weight', 'model.layers.27.mlp.down_proj.linear.weight', 'model.layers.27.mlp.down_proj.lora_A.weight', 'model.layers.27.mlp.down_proj.lora_B.weight', 'model.layers.27.mlp.gate_proj.linear.weight', 'model.layers.27.mlp.gate_proj.lora_A.weight', 'model.layers.27.mlp.gate_proj.lora_B.weight', 'model.layers.27.mlp.up_proj.linear.weight', 'model.layers.27.mlp.up_proj.lora_A.weight', 'model.layers.27.mlp.up_proj.lora_B.weight', 'model.layers.27.self_attn.k_proj.linear.bias', 'model.layers.27.self_attn.k_proj.linear.weight', 'model.layers.27.self_attn.k_proj.lora_A.weight', 'model.layers.27.self_attn.k_proj.lora_B.weight', 'model.layers.27.self_attn.o_proj.linear.weight', 'model.layers.27.self_attn.o_proj.lora_A.weight', 'model.layers.27.self_attn.o_proj.lora_B.weight', 'model.layers.27.self_attn.q_proj.linear.bias', 'model.layers.27.self_attn.q_proj.linear.weight', 'model.layers.27.self_attn.q_proj.lora_A.weight', 'model.layers.27.self_attn.q_proj.lora_B.weight', 'model.layers.27.self_attn.v_proj.linear.bias', 'model.layers.27.self_attn.v_proj.linear.weight', 'model.layers.27.self_attn.v_proj.lora_A.weight', 'model.layers.27.self_attn.v_proj.lora_B.weight', 'model.layers.3.mlp.down_proj.linear.weight', 'model.layers.3.mlp.down_proj.lora_A.weight', 'model.layers.3.mlp.down_proj.lora_B.weight', 'model.layers.3.mlp.gate_proj.linear.weight', 'model.layers.3.mlp.gate_proj.lora_A.weight', 'model.layers.3.mlp.gate_proj.lora_B.weight', 'model.layers.3.mlp.up_proj.linear.weight', 'model.layers.3.mlp.up_proj.lora_A.weight', 'model.layers.3.mlp.up_proj.lora_B.weight', 'model.layers.3.self_attn.k_proj.linear.bias', 'model.layers.3.self_attn.k_proj.linear.weight', 'model.layers.3.self_attn.k_proj.lora_A.weight', 'model.layers.3.self_attn.k_proj.lora_B.weight', 'model.layers.3.self_attn.o_proj.linear.weight', 'model.layers.3.self_attn.o_proj.lora_A.weight', 'model.layers.3.self_attn.o_proj.lora_B.weight', 'model.layers.3.self_attn.q_proj.linear.bias', 'model.layers.3.self_attn.q_proj.linear.weight', 'model.layers.3.self_attn.q_proj.lora_A.weight', 'model.layers.3.self_attn.q_proj.lora_B.weight', 'model.layers.3.self_attn.v_proj.linear.bias', 'model.layers.3.self_attn.v_proj.linear.weight', 'model.layers.3.self_attn.v_proj.lora_A.weight', 'model.layers.3.self_attn.v_proj.lora_B.weight', 'model.layers.4.mlp.down_proj.linear.weight', 'model.layers.4.mlp.down_proj.lora_A.weight', 'model.layers.4.mlp.down_proj.lora_B.weight', 'model.layers.4.mlp.gate_proj.linear.weight', 'model.layers.4.mlp.gate_proj.lora_A.weight', 'model.layers.4.mlp.gate_proj.lora_B.weight', 'model.layers.4.mlp.up_proj.linear.weight', 'model.layers.4.mlp.up_proj.lora_A.weight', 'model.layers.4.mlp.up_proj.lora_B.weight', 'model.layers.4.self_attn.k_proj.linear.bias', 'model.layers.4.self_attn.k_proj.linear.weight', 'model.layers.4.self_attn.k_proj.lora_A.weight', 'model.layers.4.self_attn.k_proj.lora_B.weight', 'model.layers.4.self_attn.o_proj.linear.weight', 'model.layers.4.self_attn.o_proj.lora_A.weight', 'model.layers.4.self_attn.o_proj.lora_B.weight', 'model.layers.4.self_attn.q_proj.linear.bias', 'model.layers.4.self_attn.q_proj.linear.weight', 'model.layers.4.self_attn.q_proj.lora_A.weight', 'model.layers.4.self_attn.q_proj.lora_B.weight', 'model.layers.4.self_attn.v_proj.linear.bias', 'model.layers.4.self_attn.v_proj.linear.weight', 'model.layers.4.self_attn.v_proj.lora_A.weight', 'model.layers.4.self_attn.v_proj.lora_B.weight', 'model.layers.5.mlp.down_proj.linear.weight', 'model.layers.5.mlp.down_proj.lora_A.weight', 'model.layers.5.mlp.down_proj.lora_B.weight', 'model.layers.5.mlp.gate_proj.linear.weight', 'model.layers.5.mlp.gate_proj.lora_A.weight', 'model.layers.5.mlp.gate_proj.lora_B.weight', 'model.layers.5.mlp.up_proj.linear.weight', 'model.layers.5.mlp.up_proj.lora_A.weight', 'model.layers.5.mlp.up_proj.lora_B.weight', 'model.layers.5.self_attn.k_proj.linear.bias', 'model.layers.5.self_attn.k_proj.linear.weight', 'model.layers.5.self_attn.k_proj.lora_A.weight', 'model.layers.5.self_attn.k_proj.lora_B.weight', 'model.layers.5.self_attn.o_proj.linear.weight', 'model.layers.5.self_attn.o_proj.lora_A.weight', 'model.layers.5.self_attn.o_proj.lora_B.weight', 'model.layers.5.self_attn.q_proj.linear.bias', 'model.layers.5.self_attn.q_proj.linear.weight', 'model.layers.5.self_attn.q_proj.lora_A.weight', 'model.layers.5.self_attn.q_proj.lora_B.weight', 'model.layers.5.self_attn.v_proj.linear.bias', 'model.layers.5.self_attn.v_proj.linear.weight', 'model.layers.5.self_attn.v_proj.lora_A.weight', 'model.layers.5.self_attn.v_proj.lora_B.weight', 'model.layers.6.mlp.down_proj.linear.weight', 'model.layers.6.mlp.down_proj.lora_A.weight', 'model.layers.6.mlp.down_proj.lora_B.weight', 'model.layers.6.mlp.gate_proj.linear.weight', 'model.layers.6.mlp.gate_proj.lora_A.weight', 'model.layers.6.mlp.gate_proj.lora_B.weight', 'model.layers.6.mlp.up_proj.linear.weight', 'model.layers.6.mlp.up_proj.lora_A.weight', 'model.layers.6.mlp.up_proj.lora_B.weight', 'model.layers.6.self_attn.k_proj.linear.bias', 'model.layers.6.self_attn.k_proj.linear.weight', 'model.layers.6.self_attn.k_proj.lora_A.weight', 'model.layers.6.self_attn.k_proj.lora_B.weight', 'model.layers.6.self_attn.o_proj.linear.weight', 'model.layers.6.self_attn.o_proj.lora_A.weight', 'model.layers.6.self_attn.o_proj.lora_B.weight', 'model.layers.6.self_attn.q_proj.linear.bias', 'model.layers.6.self_attn.q_proj.linear.weight', 'model.layers.6.self_attn.q_proj.lora_A.weight', 'model.layers.6.self_attn.q_proj.lora_B.weight', 'model.layers.6.self_attn.v_proj.linear.bias', 'model.layers.6.self_attn.v_proj.linear.weight', 'model.layers.6.self_attn.v_proj.lora_A.weight', 'model.layers.6.self_attn.v_proj.lora_B.weight', 'model.layers.7.mlp.down_proj.linear.weight', 'model.layers.7.mlp.down_proj.lora_A.weight', 'model.layers.7.mlp.down_proj.lora_B.weight', 'model.layers.7.mlp.gate_proj.linear.weight', 'model.layers.7.mlp.gate_proj.lora_A.weight', 'model.layers.7.mlp.gate_proj.lora_B.weight', 'model.layers.7.mlp.up_proj.linear.weight', 'model.layers.7.mlp.up_proj.lora_A.weight', 'model.layers.7.mlp.up_proj.lora_B.weight', 'model.layers.7.self_attn.k_proj.linear.bias', 'model.layers.7.self_attn.k_proj.linear.weight', 'model.layers.7.self_attn.k_proj.lora_A.weight', 'model.layers.7.self_attn.k_proj.lora_B.weight', 'model.layers.7.self_attn.o_proj.linear.weight', 'model.layers.7.self_attn.o_proj.lora_A.weight', 'model.layers.7.self_attn.o_proj.lora_B.weight', 'model.layers.7.self_attn.q_proj.linear.bias', 'model.layers.7.self_attn.q_proj.linear.weight', 'model.layers.7.self_attn.q_proj.lora_A.weight', 'model.layers.7.self_attn.q_proj.lora_B.weight', 'model.layers.7.self_attn.v_proj.linear.bias', 'model.layers.7.self_attn.v_proj.linear.weight', 'model.layers.7.self_attn.v_proj.lora_A.weight', 'model.layers.7.self_attn.v_proj.lora_B.weight', 'model.layers.8.mlp.down_proj.linear.weight', 'model.layers.8.mlp.down_proj.lora_A.weight', 'model.layers.8.mlp.down_proj.lora_B.weight', 'model.layers.8.mlp.gate_proj.linear.weight', 'model.layers.8.mlp.gate_proj.lora_A.weight', 'model.layers.8.mlp.gate_proj.lora_B.weight', 'model.layers.8.mlp.up_proj.linear.weight', 'model.layers.8.mlp.up_proj.lora_A.weight', 'model.layers.8.mlp.up_proj.lora_B.weight', 'model.layers.8.self_attn.k_proj.linear.bias', 'model.layers.8.self_attn.k_proj.linear.weight', 'model.layers.8.self_attn.k_proj.lora_A.weight', 'model.layers.8.self_attn.k_proj.lora_B.weight', 'model.layers.8.self_attn.o_proj.linear.weight', 'model.layers.8.self_attn.o_proj.lora_A.weight', 'model.layers.8.self_attn.o_proj.lora_B.weight', 'model.layers.8.self_attn.q_proj.linear.bias', 'model.layers.8.self_attn.q_proj.linear.weight', 'model.layers.8.self_attn.q_proj.lora_A.weight', 'model.layers.8.self_attn.q_proj.lora_B.weight', 'model.layers.8.self_attn.v_proj.linear.bias', 'model.layers.8.self_attn.v_proj.linear.weight', 'model.layers.8.self_attn.v_proj.lora_A.weight', 'model.layers.8.self_attn.v_proj.lora_B.weight', 'model.layers.9.mlp.down_proj.linear.weight', 'model.layers.9.mlp.down_proj.lora_A.weight', 'model.layers.9.mlp.down_proj.lora_B.weight', 'model.layers.9.mlp.gate_proj.linear.weight', 'model.layers.9.mlp.gate_proj.lora_A.weight', 'model.layers.9.mlp.gate_proj.lora_B.weight', 'model.layers.9.mlp.up_proj.linear.weight', 'model.layers.9.mlp.up_proj.lora_A.weight', 'model.layers.9.mlp.up_proj.lora_B.weight', 'model.layers.9.self_attn.k_proj.linear.bias', 'model.layers.9.self_attn.k_proj.linear.weight', 'model.layers.9.self_attn.k_proj.lora_A.weight', 'model.layers.9.self_attn.k_proj.lora_B.weight', 'model.layers.9.self_attn.o_proj.linear.weight', 'model.layers.9.self_attn.o_proj.lora_A.weight', 'model.layers.9.self_attn.o_proj.lora_B.weight', 'model.layers.9.self_attn.q_proj.linear.bias', 'model.layers.9.self_attn.q_proj.linear.weight', 'model.layers.9.self_attn.q_proj.lora_A.weight', 'model.layers.9.self_attn.q_proj.lora_B.weight', 'model.layers.9.self_attn.v_proj.linear.bias', 'model.layers.9.self_attn.v_proj.linear.weight', 'model.layers.9.self_attn.v_proj.lora_A.weight', 'model.layers.9.self_attn.v_proj.lora_B.weight']\n",
      "- This IS expected if you are initializing Model from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing Model from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of Model were not initialized from the model checkpoint at lora-embedding-64-audio-qwen2.5-7b-malaysian-10k-stage2/checkpoint-300 and are newly initialized: ['encoder.range_max_source_positions', 'model.layers.0.mlp.down_proj.weight', 'model.layers.0.mlp.gate_proj.weight', 'model.layers.0.mlp.up_proj.weight', 'model.layers.0.self_attn.k_proj.bias', 'model.layers.0.self_attn.k_proj.weight', 'model.layers.0.self_attn.o_proj.weight', 'model.layers.0.self_attn.q_proj.bias', 'model.layers.0.self_attn.q_proj.weight', 'model.layers.0.self_attn.v_proj.bias', 'model.layers.0.self_attn.v_proj.weight', 'model.layers.1.mlp.down_proj.weight', 'model.layers.1.mlp.gate_proj.weight', 'model.layers.1.mlp.up_proj.weight', 'model.layers.1.self_attn.k_proj.bias', 'model.layers.1.self_attn.k_proj.weight', 'model.layers.1.self_attn.o_proj.weight', 'model.layers.1.self_attn.q_proj.bias', 'model.layers.1.self_attn.q_proj.weight', 'model.layers.1.self_attn.v_proj.bias', 'model.layers.1.self_attn.v_proj.weight', 'model.layers.10.mlp.down_proj.weight', 'model.layers.10.mlp.gate_proj.weight', 'model.layers.10.mlp.up_proj.weight', 'model.layers.10.self_attn.k_proj.bias', 'model.layers.10.self_attn.k_proj.weight', 'model.layers.10.self_attn.o_proj.weight', 'model.layers.10.self_attn.q_proj.bias', 'model.layers.10.self_attn.q_proj.weight', 'model.layers.10.self_attn.v_proj.bias', 'model.layers.10.self_attn.v_proj.weight', 'model.layers.11.mlp.down_proj.weight', 'model.layers.11.mlp.gate_proj.weight', 'model.layers.11.mlp.up_proj.weight', 'model.layers.11.self_attn.k_proj.bias', 'model.layers.11.self_attn.k_proj.weight', 'model.layers.11.self_attn.o_proj.weight', 'model.layers.11.self_attn.q_proj.bias', 'model.layers.11.self_attn.q_proj.weight', 'model.layers.11.self_attn.v_proj.bias', 'model.layers.11.self_attn.v_proj.weight', 'model.layers.12.mlp.down_proj.weight', 'model.layers.12.mlp.gate_proj.weight', 'model.layers.12.mlp.up_proj.weight', 'model.layers.12.self_attn.k_proj.bias', 'model.layers.12.self_attn.k_proj.weight', 'model.layers.12.self_attn.o_proj.weight', 'model.layers.12.self_attn.q_proj.bias', 'model.layers.12.self_attn.q_proj.weight', 'model.layers.12.self_attn.v_proj.bias', 'model.layers.12.self_attn.v_proj.weight', 'model.layers.13.mlp.down_proj.weight', 'model.layers.13.mlp.gate_proj.weight', 'model.layers.13.mlp.up_proj.weight', 'model.layers.13.self_attn.k_proj.bias', 'model.layers.13.self_attn.k_proj.weight', 'model.layers.13.self_attn.o_proj.weight', 'model.layers.13.self_attn.q_proj.bias', 'model.layers.13.self_attn.q_proj.weight', 'model.layers.13.self_attn.v_proj.bias', 'model.layers.13.self_attn.v_proj.weight', 'model.layers.14.mlp.down_proj.weight', 'model.layers.14.mlp.gate_proj.weight', 'model.layers.14.mlp.up_proj.weight', 'model.layers.14.self_attn.k_proj.bias', 'model.layers.14.self_attn.k_proj.weight', 'model.layers.14.self_attn.o_proj.weight', 'model.layers.14.self_attn.q_proj.bias', 'model.layers.14.self_attn.q_proj.weight', 'model.layers.14.self_attn.v_proj.bias', 'model.layers.14.self_attn.v_proj.weight', 'model.layers.15.mlp.down_proj.weight', 'model.layers.15.mlp.gate_proj.weight', 'model.layers.15.mlp.up_proj.weight', 'model.layers.15.self_attn.k_proj.bias', 'model.layers.15.self_attn.k_proj.weight', 'model.layers.15.self_attn.o_proj.weight', 'model.layers.15.self_attn.q_proj.bias', 'model.layers.15.self_attn.q_proj.weight', 'model.layers.15.self_attn.v_proj.bias', 'model.layers.15.self_attn.v_proj.weight', 'model.layers.16.mlp.down_proj.weight', 'model.layers.16.mlp.gate_proj.weight', 'model.layers.16.mlp.up_proj.weight', 'model.layers.16.self_attn.k_proj.bias', 'model.layers.16.self_attn.k_proj.weight', 'model.layers.16.self_attn.o_proj.weight', 'model.layers.16.self_attn.q_proj.bias', 'model.layers.16.self_attn.q_proj.weight', 'model.layers.16.self_attn.v_proj.bias', 'model.layers.16.self_attn.v_proj.weight', 'model.layers.17.mlp.down_proj.weight', 'model.layers.17.mlp.gate_proj.weight', 'model.layers.17.mlp.up_proj.weight', 'model.layers.17.self_attn.k_proj.bias', 'model.layers.17.self_attn.k_proj.weight', 'model.layers.17.self_attn.o_proj.weight', 'model.layers.17.self_attn.q_proj.bias', 'model.layers.17.self_attn.q_proj.weight', 'model.layers.17.self_attn.v_proj.bias', 'model.layers.17.self_attn.v_proj.weight', 'model.layers.18.mlp.down_proj.weight', 'model.layers.18.mlp.gate_proj.weight', 'model.layers.18.mlp.up_proj.weight', 'model.layers.18.self_attn.k_proj.bias', 'model.layers.18.self_attn.k_proj.weight', 'model.layers.18.self_attn.o_proj.weight', 'model.layers.18.self_attn.q_proj.bias', 'model.layers.18.self_attn.q_proj.weight', 'model.layers.18.self_attn.v_proj.bias', 'model.layers.18.self_attn.v_proj.weight', 'model.layers.19.mlp.down_proj.weight', 'model.layers.19.mlp.gate_proj.weight', 'model.layers.19.mlp.up_proj.weight', 'model.layers.19.self_attn.k_proj.bias', 'model.layers.19.self_attn.k_proj.weight', 'model.layers.19.self_attn.o_proj.weight', 'model.layers.19.self_attn.q_proj.bias', 'model.layers.19.self_attn.q_proj.weight', 'model.layers.19.self_attn.v_proj.bias', 'model.layers.19.self_attn.v_proj.weight', 'model.layers.2.mlp.down_proj.weight', 'model.layers.2.mlp.gate_proj.weight', 'model.layers.2.mlp.up_proj.weight', 'model.layers.2.self_attn.k_proj.bias', 'model.layers.2.self_attn.k_proj.weight', 'model.layers.2.self_attn.o_proj.weight', 'model.layers.2.self_attn.q_proj.bias', 'model.layers.2.self_attn.q_proj.weight', 'model.layers.2.self_attn.v_proj.bias', 'model.layers.2.self_attn.v_proj.weight', 'model.layers.20.mlp.down_proj.weight', 'model.layers.20.mlp.gate_proj.weight', 'model.layers.20.mlp.up_proj.weight', 'model.layers.20.self_attn.k_proj.bias', 'model.layers.20.self_attn.k_proj.weight', 'model.layers.20.self_attn.o_proj.weight', 'model.layers.20.self_attn.q_proj.bias', 'model.layers.20.self_attn.q_proj.weight', 'model.layers.20.self_attn.v_proj.bias', 'model.layers.20.self_attn.v_proj.weight', 'model.layers.21.mlp.down_proj.weight', 'model.layers.21.mlp.gate_proj.weight', 'model.layers.21.mlp.up_proj.weight', 'model.layers.21.self_attn.k_proj.bias', 'model.layers.21.self_attn.k_proj.weight', 'model.layers.21.self_attn.o_proj.weight', 'model.layers.21.self_attn.q_proj.bias', 'model.layers.21.self_attn.q_proj.weight', 'model.layers.21.self_attn.v_proj.bias', 'model.layers.21.self_attn.v_proj.weight', 'model.layers.22.mlp.down_proj.weight', 'model.layers.22.mlp.gate_proj.weight', 'model.layers.22.mlp.up_proj.weight', 'model.layers.22.self_attn.k_proj.bias', 'model.layers.22.self_attn.k_proj.weight', 'model.layers.22.self_attn.o_proj.weight', 'model.layers.22.self_attn.q_proj.bias', 'model.layers.22.self_attn.q_proj.weight', 'model.layers.22.self_attn.v_proj.bias', 'model.layers.22.self_attn.v_proj.weight', 'model.layers.23.mlp.down_proj.weight', 'model.layers.23.mlp.gate_proj.weight', 'model.layers.23.mlp.up_proj.weight', 'model.layers.23.self_attn.k_proj.bias', 'model.layers.23.self_attn.k_proj.weight', 'model.layers.23.self_attn.o_proj.weight', 'model.layers.23.self_attn.q_proj.bias', 'model.layers.23.self_attn.q_proj.weight', 'model.layers.23.self_attn.v_proj.bias', 'model.layers.23.self_attn.v_proj.weight', 'model.layers.24.mlp.down_proj.weight', 'model.layers.24.mlp.gate_proj.weight', 'model.layers.24.mlp.up_proj.weight', 'model.layers.24.self_attn.k_proj.bias', 'model.layers.24.self_attn.k_proj.weight', 'model.layers.24.self_attn.o_proj.weight', 'model.layers.24.self_attn.q_proj.bias', 'model.layers.24.self_attn.q_proj.weight', 'model.layers.24.self_attn.v_proj.bias', 'model.layers.24.self_attn.v_proj.weight', 'model.layers.25.mlp.down_proj.weight', 'model.layers.25.mlp.gate_proj.weight', 'model.layers.25.mlp.up_proj.weight', 'model.layers.25.self_attn.k_proj.bias', 'model.layers.25.self_attn.k_proj.weight', 'model.layers.25.self_attn.o_proj.weight', 'model.layers.25.self_attn.q_proj.bias', 'model.layers.25.self_attn.q_proj.weight', 'model.layers.25.self_attn.v_proj.bias', 'model.layers.25.self_attn.v_proj.weight', 'model.layers.26.mlp.down_proj.weight', 'model.layers.26.mlp.gate_proj.weight', 'model.layers.26.mlp.up_proj.weight', 'model.layers.26.self_attn.k_proj.bias', 'model.layers.26.self_attn.k_proj.weight', 'model.layers.26.self_attn.o_proj.weight', 'model.layers.26.self_attn.q_proj.bias', 'model.layers.26.self_attn.q_proj.weight', 'model.layers.26.self_attn.v_proj.bias', 'model.layers.26.self_attn.v_proj.weight', 'model.layers.27.mlp.down_proj.weight', 'model.layers.27.mlp.gate_proj.weight', 'model.layers.27.mlp.up_proj.weight', 'model.layers.27.self_attn.k_proj.bias', 'model.layers.27.self_attn.k_proj.weight', 'model.layers.27.self_attn.o_proj.weight', 'model.layers.27.self_attn.q_proj.bias', 'model.layers.27.self_attn.q_proj.weight', 'model.layers.27.self_attn.v_proj.bias', 'model.layers.27.self_attn.v_proj.weight', 'model.layers.3.mlp.down_proj.weight', 'model.layers.3.mlp.gate_proj.weight', 'model.layers.3.mlp.up_proj.weight', 'model.layers.3.self_attn.k_proj.bias', 'model.layers.3.self_attn.k_proj.weight', 'model.layers.3.self_attn.o_proj.weight', 'model.layers.3.self_attn.q_proj.bias', 'model.layers.3.self_attn.q_proj.weight', 'model.layers.3.self_attn.v_proj.bias', 'model.layers.3.self_attn.v_proj.weight', 'model.layers.4.mlp.down_proj.weight', 'model.layers.4.mlp.gate_proj.weight', 'model.layers.4.mlp.up_proj.weight', 'model.layers.4.self_attn.k_proj.bias', 'model.layers.4.self_attn.k_proj.weight', 'model.layers.4.self_attn.o_proj.weight', 'model.layers.4.self_attn.q_proj.bias', 'model.layers.4.self_attn.q_proj.weight', 'model.layers.4.self_attn.v_proj.bias', 'model.layers.4.self_attn.v_proj.weight', 'model.layers.5.mlp.down_proj.weight', 'model.layers.5.mlp.gate_proj.weight', 'model.layers.5.mlp.up_proj.weight', 'model.layers.5.self_attn.k_proj.bias', 'model.layers.5.self_attn.k_proj.weight', 'model.layers.5.self_attn.o_proj.weight', 'model.layers.5.self_attn.q_proj.bias', 'model.layers.5.self_attn.q_proj.weight', 'model.layers.5.self_attn.v_proj.bias', 'model.layers.5.self_attn.v_proj.weight', 'model.layers.6.mlp.down_proj.weight', 'model.layers.6.mlp.gate_proj.weight', 'model.layers.6.mlp.up_proj.weight', 'model.layers.6.self_attn.k_proj.bias', 'model.layers.6.self_attn.k_proj.weight', 'model.layers.6.self_attn.o_proj.weight', 'model.layers.6.self_attn.q_proj.bias', 'model.layers.6.self_attn.q_proj.weight', 'model.layers.6.self_attn.v_proj.bias', 'model.layers.6.self_attn.v_proj.weight', 'model.layers.7.mlp.down_proj.weight', 'model.layers.7.mlp.gate_proj.weight', 'model.layers.7.mlp.up_proj.weight', 'model.layers.7.self_attn.k_proj.bias', 'model.layers.7.self_attn.k_proj.weight', 'model.layers.7.self_attn.o_proj.weight', 'model.layers.7.self_attn.q_proj.bias', 'model.layers.7.self_attn.q_proj.weight', 'model.layers.7.self_attn.v_proj.bias', 'model.layers.7.self_attn.v_proj.weight', 'model.layers.8.mlp.down_proj.weight', 'model.layers.8.mlp.gate_proj.weight', 'model.layers.8.mlp.up_proj.weight', 'model.layers.8.self_attn.k_proj.bias', 'model.layers.8.self_attn.k_proj.weight', 'model.layers.8.self_attn.o_proj.weight', 'model.layers.8.self_attn.q_proj.bias', 'model.layers.8.self_attn.q_proj.weight', 'model.layers.8.self_attn.v_proj.bias', 'model.layers.8.self_attn.v_proj.weight', 'model.layers.9.mlp.down_proj.weight', 'model.layers.9.mlp.gate_proj.weight', 'model.layers.9.mlp.up_proj.weight', 'model.layers.9.self_attn.k_proj.bias', 'model.layers.9.self_attn.k_proj.weight', 'model.layers.9.self_attn.o_proj.weight', 'model.layers.9.self_attn.q_proj.bias', 'model.layers.9.self_attn.q_proj.weight', 'model.layers.9.self_attn.v_proj.bias', 'model.layers.9.self_attn.v_proj.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "model = Model.from_pretrained(checkpoint)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "2146004e",
   "metadata": {},
   "outputs": [],
   "source": [
    "state_dict = model.state_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "53977df7",
   "metadata": {},
   "outputs": [],
   "source": [
    "from glob import glob\n",
    "\n",
    "files = glob(f'{checkpoint}/*.safetensors')\n",
    "tensors = {}\n",
    "for f in files:\n",
    "    with safe_open(f, framework=\"pt\", device='cpu') as f:\n",
    "        for k in f.keys():\n",
    "            tensors[k] = f.get_tensor(k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "4ad8d0ee",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████| 1219/1219 [00:03<00:00, 398.95it/s]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "for k in tqdm(tensors):\n",
    "    if 'linear.weight' in k:\n",
    "        k_ori = k\n",
    "        k = k.replace('linear.weight', 'weight')\n",
    "        W = state_dict[k]\n",
    "        A = tensors[k_ori].type(W.dtype)\n",
    "        W.copy_(A)\n",
    "    \n",
    "    if 'linear.bias' in k:\n",
    "        k_ori = k\n",
    "        k = k.replace('linear.bias', 'bias')\n",
    "        W = state_dict[k]\n",
    "        A = tensors[k_ori].type(W.dtype)\n",
    "        W.copy_(A)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "6a146ec6",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████| 196/196 [00:00<00:00, 406.70it/s]\n"
     ]
    }
   ],
   "source": [
    "keys = tensors.keys()\n",
    "keys_lora = sorted(list(set([k.split('.lora')[0] for k in keys if '.lora' in k])))\n",
    "for k in tqdm(keys_lora):\n",
    "    k_ori = k + '.weight'\n",
    "    post_A = '.lora_A.weight'\n",
    "    post_B = '.lora_B.weight'\n",
    "    A = k + post_A\n",
    "    B = k + post_B\n",
    "    W = state_dict[k_ori]\n",
    "    A = tensors[A].type(W.dtype)\n",
    "    B = tensors[B].type(W.dtype)\n",
    "    W = W.t()\n",
    "    with torch.no_grad():\n",
    "        W.addmm_(A.t(), B.t(), alpha = 2.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "459b190c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7d34cc0a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "with open('prepare-Speaker-Diarization-Instructions.json') as fopen:\n",
    "    instructions = json.load(fopen)\n",
    "len(instructions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "79e64d45",
   "metadata": {},
   "outputs": [],
   "source": [
    "import IPython.display as ipd\n",
    "ipd.Audio(instructions[0]['audio'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "024f17de",
   "metadata": {},
   "outputs": [],
   "source": [
    "f = 'coding-dockerfile (1).mp3'\n",
    "audio_ = audio_class.decode_example(audio_class.encode_example(f))['array']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "ddb6e65c",
   "metadata": {},
   "outputs": [],
   "source": [
    "audio_class = Audio(sampling_rate=16000)\n",
    "\n",
    "def process(messages):\n",
    "    audios = []\n",
    "    for message in messages:\n",
    "        if isinstance(message[\"content\"], list):\n",
    "            for ele in message[\"content\"]:\n",
    "                if ele[\"type\"] == \"audio\":\n",
    "                    audios.append(ele['audio_url'])\n",
    "    \n",
    "    y = []\n",
    "    audio_length = 0\n",
    "    for f in audios:\n",
    "        y_ = audio_class.decode_example(audio_class.encode_example(f))['array']\n",
    "        y.append(y_)\n",
    "        audio_length += min(3000, math.ceil(len(y_) / feature_extractor.hop_length))\n",
    "    audio_length = (audio_length - 1) // 2 + 1\n",
    "\n",
    "    text = tokenizer.apply_chat_template(messages, add_generation_prompt=True, tokenize=False)\n",
    "    expanded_audio_token = audio_token * audio_length\n",
    "    text = text.replace(audio_token, expanded_audio_token)\n",
    "    inputs = tokenizer(text, return_tensors = 'pt')\n",
    "    input_ids = inputs['input_ids']\n",
    "    inputs_audio = feature_extractor(\n",
    "        y, \n",
    "        return_attention_mask=True, \n",
    "        padding=\"max_length\", \n",
    "        sampling_rate=16000,\n",
    "        return_tensors = 'pt'\n",
    "    )\n",
    "    input_features = inputs_audio['input_features']\n",
    "    feature_attention_mask = inputs_audio['attention_mask']\n",
    "    \n",
    "    return {\n",
    "        'input_ids': input_ids, \n",
    "        'input_features': input_features,\n",
    "        'feature_attention_mask': feature_attention_mask,\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "8909f0b0",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "The attention mask is not set and cannot be inferred from input because pad token is same as eos token. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<|im_start|>system\n",
      "You are a helpful assistant.<|im_end|>\n",
      "<|im_start|>user\n",
      "Audio 1: <|audio_bos|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|AUDIO|><|audio_eos|>\n",
      "transcribe the audio<|im_end|>\n",
      "<|im_start|>assistant\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Zamly on aging di Vienna Austria yang telah diadakan pada tahun 1982 dan berasaskan unjuran tersebut, maka Jabatan Perangkaan Malaysia menganggarkan menjelang tahun 2035, sejumlah 15 peratus penduduk kita adalah daripada kalangan warga emas. Untuk makluman Tuan Yang di-Pertua dan juga Aliam Booyat, pembangunan sistem pendaftaran warga emas ataupun kita sebutkan E-VEN adalah usaha kerajaan ke arah merealisasikan\n"
     ]
    }
   ],
   "source": [
    "messages = [\n",
    "    {\"role\": \"user\", \"content\": [\n",
    "        {\"type\": \"audio\", \"audio_url\": 'assembly-1.mp3'},\n",
    "        {\"type\": \"text\", \"text\": \"transcribe the audio\"},\n",
    "    ]},\n",
    "]\n",
    "\n",
    "inputs = process(messages)\n",
    "inputs\n",
    "with torch.no_grad():\n",
    "    generate_kwargs = dict(\n",
    "        **inputs,\n",
    "        max_new_tokens=128,\n",
    "        top_p=0.95,\n",
    "        top_k=50,\n",
    "        temperature=0.9,\n",
    "        do_sample=True,\n",
    "        repetition_penalty=1.05,\n",
    "        streamer=streamer\n",
    "    )\n",
    "    generation_output = model.generate(**generate_kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "76ddf0f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "conversation = [\n",
    "    {\"role\": \"user\", \"content\": [\n",
    "        {\"type\": \"audio\", \"audio_url\": \"audio.wav\"},\n",
    "        {\"type\": \"text\", \"text\": \"transcribe bunyi dlm format srt\"},\n",
    "    ]},\n",
    "]\n",
    "text = tokenizer.apply_chat_template(conversation, add_generation_prompt=True, tokenize=False)\n",
    "audio_lengths = [min(3000, math.ceil(len(audio_) / processor.feature_extractor.hop_length))]\n",
    "audio_length = audio_lengths.pop(0)\n",
    "input_length = (audio_length - 1) // 2 + 1\n",
    "\n",
    "expanded_audio_token = audio_token * input_length\n",
    "text = text.replace(audio_token, expanded_audio_token)\n",
    "inputs = tokenizer(text, return_tensors = 'pt')\n",
    "input_ids = inputs['input_ids']\n",
    "inputs_audio = processor.feature_extractor(\n",
    "    [audio_], \n",
    "    return_attention_mask=True, \n",
    "    padding=\"max_length\", \n",
    "    sampling_rate=16000,\n",
    "    return_tensors = 'pt'\n",
    ")\n",
    "\n",
    "input_features = inputs_audio['input_features']\n",
    "feature_attention_mask = inputs_audio['attention_mask']\n",
    "with torch.no_grad():\n",
    "    generate_kwargs = dict(\n",
    "        input_ids = input_ids, \n",
    "        input_features = input_features,\n",
    "        feature_attention_mask = feature_attention_mask,\n",
    "        max_new_tokens=128,\n",
    "        top_p=0.95,\n",
    "        top_k=50,\n",
    "        temperature=0.9,\n",
    "        do_sample=True,\n",
    "        repetition_penalty=1.05,\n",
    "        streamer=streamer\n",
    "    )\n",
    "    generation_output = model.generate(**generate_kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "b2cc888d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<|im_start|>system\n",
      "anda adalah chatbot untuk Maybank<|im_end|>\n",
      "<|im_start|>user\n",
      "hello sye hilang credit card saye<|im_end|>\n",
      "<|im_start|>assistant\n",
      "Saya minta maaf atas kehilangan kad kredit anda. Bolehkah saya mendapatkan butiran mengenai situasi ini?<|im_end|>\n"
     ]
    }
   ],
   "source": [
    "conversation = [\n",
    "    {'role': 'system', 'content': 'anda adalah chatbot untuk Maybank'},\n",
    "    {\"role\": \"user\", \"content\": [\n",
    "        {\"type\": \"text\", \"text\": \"hello sye hilang credit card saye\"},\n",
    "    ]},\n",
    "]\n",
    "text = tokenizer.apply_chat_template(conversation, add_generation_prompt=True, tokenize=False)\n",
    "inputs = tokenizer(text, return_tensors = 'pt')\n",
    "input_ids = inputs['input_ids']\n",
    "with torch.no_grad():\n",
    "    generate_kwargs = dict(\n",
    "        input_ids = input_ids, \n",
    "        max_new_tokens=128,\n",
    "        top_p=0.95,\n",
    "        top_k=50,\n",
    "        temperature=0.9,\n",
    "        do_sample=True,\n",
    "        repetition_penalty=1.05,\n",
    "        streamer=streamer\n",
    "    )\n",
    "    generation_output = model.generate(**generate_kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "fc1cebc7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('./7b-audio/tokenizer_config.json',\n",
       " './7b-audio/special_tokens_map.json',\n",
       " './7b-audio/vocab.json',\n",
       " './7b-audio/merges.txt',\n",
       " './7b-audio/added_tokens.json',\n",
       " './7b-audio/tokenizer.json')"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.to(torch.bfloat16).save_pretrained('./7b-audio')\n",
    "feature_extractor.save_pretrained('./7b-audio')\n",
    "tokenizer.save_pretrained('./7b-audio')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "1ab62853",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start hashing 16 files.\n",
      "Finished hashing 16 files.\n",
      "Uploading files using Xet Storage..\n",
      "Uploading...: 100%|████████████████████████| 16.5G/16.5G [03:05<00:00, 89.0MB/s]\n",
      "Removing 12 file(s) from commit that have not changed.\n",
      "https://huggingface.co/mesolitica/Malaysian-Audio-Qwen2.5-7B-Instruct/tree/stage2/.\n"
     ]
    }
   ],
   "source": [
    "!HF_HUB_ENABLE_HF_TRANSFER=\"1\" huggingface-cli upload mesolitica/Malaysian-Audio-Qwen2.5-7B-Instruct ./7b-audio . --revision=\"stage2\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "87e787e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eb97e865",
   "metadata": {},
   "outputs": [],
   "source": [
    "!HF_HUB_ENABLE_HF_TRANSFER=\"1\" huggingface-cli upload --help"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "35825629",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
